package main

import (
	"bufio"
	"bytes"
	"code.google.com/p/go.text/encoding/simplifiedchinese"
	"code.google.com/p/go.text/transform"
	"fmt"
	goq "github.com/PuerkitoBio/goquery"
	_ "github.com/go-sql-driver/mysql"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	//"strconv"
	"database/sql"
	"runtime"
	"strings"
	"sync"
	"time"
)

var (
	main_domain = "" //主域名
	waitgroup   = &sync.WaitGroup{}
	LOG         = "baidu_brand"
	ErrLOG      = "baidu_brand"
)

func main() {
	main_domain = "https://www.baidu.com"
	LOG = "bj." + LOG
	ErrLOG = "bj." + ErrLOG

	getKeywordList()
	fmt.Println("done.")
}

//获取数据源
func getDB() string {
	file, e := os.OpenFile("config.ini", os.O_RDONLY, 0400)
	defer file.Close()
	if e != nil {
		return ""
	}
	rd := bufio.NewReader(file)
	buf := make([]byte, 512)
	buf, _, e = rd.ReadLine()
	file.Read(buf)
	if e != nil {
		return ""
	}
	return string(buf[:])
}

//取关键词列表
func getKeywordList() {
	dbdriver := getDB()
	db, err := sql.Open("mysql", dbdriver)
	if err != nil {
		errlog(err.Error())
		return
	}
	rows, err := db.Query("select communityname from es_home_baseinfo where status=1")
	if err != nil {
		errlog(err.Error())
		return
	}
	var communityname string
	var i = 0
	map_list := make(map[int]string)
	for rows.Next() {
		err := rows.Scan(&communityname)
		if err != nil {
			errlog(err.Error())
			continue
		}
		communityname = G2U(communityname)
		map_list[i] = communityname
		i++
	}
	for i = 0; i < len(map_list); i += runtime.NumCPU() {
		for j := i; j < i+runtime.NumCPU() && j < len(map_list); j++ {
			waitgroup.Add(1)
			go getKeyword(j, map_list[j])
		}
		waitgroup.Wait()
	}
	rows.Close()
	defer db.Close()
}

//获取百度上面此关键字在不在
func getKeyword(j int, url string) {
	defer waitgroup.Done()
	tmpurl := fmt.Sprintf("%s/s?wd=%s&rsv_spt=1&issp=1&f=8&rsv_bp=1&rsv_idx=2&ie=utf-8&tn=baiduhome_pg&rsv_enter=1&rsv_pq=945d664b00012b5d&rsv_t=bd1ctHlw82fp7EXkx7HbzWTEOq0h5CsuPRzjt0L93cZt4h0NXNX3BmQkPKELFTEHHghb&inputT=2991218&rsv_sug3=31&rsv_sug1=11", main_domain, url)
	doc, e := getDoc(tmpurl)
	if e != nil {
		errlog(tmpurl + " ::" + e.Error())
		return
	}
	ret := doc.Find("a:contains('交通便捷生活便利 新浪二手房')").Text()
	if ret != "" {
		outlog(fmt.Sprintf("%d||%s||在线", j, url))
	} else {
		outlog(fmt.Sprintf("%d||%s||不在线", j, url))
	}
}

//获取doc
func getDoc(url string) (*goq.Document, error) {
	res, e := getResp(url)
	if e != nil {
		return nil, e
	}
	doc, e := goq.NewDocumentFromResponse(res)
	if e != nil {
		return nil, e
	}
	return doc, nil
}

//获取response
func getResp(url string) (*http.Response, error) {
	req, e := http.NewRequest("GET", url, nil)
	req.Header.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
	//req.Header.Add("Accept-Encoding", "gzip,deflate,sdch")
	req.Header.Add("Accept-Language", "zh-CN,zh;q=0.8")
	req.Header.Add("Cache-Control", "max-age=0")
	req.Header.Add("Proxy-Connection", "keep-alive")
	req.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36")
	c := &http.Client{
		Transport: &http.Transport{
			Dial: func(netw, addr string) (net.Conn, error) {
				deadline := time.Now().Add(25 * time.Second)
				c, err := net.DialTimeout(netw, addr, time.Second*20)
				if err != nil {
					return nil, err
				}
				c.SetDeadline(deadline)
				return c, nil
			},
		},
	}

	var resp *http.Response
	//尝试三次
	for retry := 0; retry < 3; retry++ {
		resp, e = c.Do(req)
		if e != nil {
			time.Sleep(time.Second * 1)
		} else {
			break
		}
	}
	if e != nil {
		return nil, e
	}
	return resp, nil
}

//gbk->utf8
func G2U(s string) string {
	ret, e := gbktoutf8([]byte(s))
	if e != nil {
		return ""
	}
	return string(ret)
}

//utf8->gbk
func U2G(s string) string {
	ret, e := utf8togbk([]byte(s))
	if e != nil {
		return ""
	}
	return string(ret)
}

func gbktoutf8(s []byte) ([]byte, error) {
	reader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewDecoder())
	d, e := ioutil.ReadAll(reader)
	if e != nil {
		return nil, e
	}
	return d, nil
}
func utf8togbk(s []byte) ([]byte, error) {
	reader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewEncoder())
	d, e := ioutil.ReadAll(reader)
	if e != nil {
		return nil, e
	}
	return d, nil
}

func filterStr(str string) string {
	str = strings.Replace(str, "\r\n", "", -1)
	str = strings.Replace(str, "\r", "", -1)
	str = strings.Replace(str, "\n", "", -1)
	str = strings.Replace(str, "|", "", -1)
	str = strings.Replace(str, ",", "", -1)
	str = strings.TrimSpace(str)
	return str
}

func s(a ...interface{}) {
	fmt.Println(a)
}

//outlog
func outlog(str string) {
	pwd, _ := os.Getwd()
	outlog := fmt.Sprintf("%s/%s_log.txt", pwd, LOG)
	file, err := os.OpenFile(outlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	defer file.Close()
	if err != nil {
		return
	}
	str += "\n"
	file.WriteString(str)
}

//outlog
func errlog(str string) {
	pwd, _ := os.Getwd()
	outlog := fmt.Sprintf("%s/%s_err.txt", pwd, ErrLOG)
	file, err := os.OpenFile(outlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	defer file.Close()
	if err != nil {
		return
	}
	str += "\n"
	file.WriteString(str)
}
